#coding:utf-8
'''
* coder  : dzlua
* email  : 505544956@qq.com
* module : tohuashun
* path   : .
* file   : tohuashun.py
* time   : 2017-11-10 09:46:55
'''
#--------------------#
import sys, codecs, os, time
from bs4 import BeautifulSoup
#----------#
sys.path.append("..")
#----------#
from spider import Spider
from database import DataBase
#--------------------#

#--------------------#
class TohuaShun(Spider, DataBase):
    def __init__(self, host, user, pwd, db, save_path, threads_work = 4, tasks = 0):
        self.__tb = 'tohuashun'

        self.__save_path = save_path

        DataBase.__init__(self, host, user, pwd, db)

        Spider.__init__(self, threads_work, tasks)

        sql = "CREATE TABLE IF NOT EXISTS " + self.__tb + " " \
            "(id INT AUTO_INCREMENT PRIMARY KEY," \
            " title TEXT," \
            " href TEXT )" \
            " DEFAULT CHARSET=utf8;"
        DataBase.execute(self, sql)

        self.__url = 'http://www.guhai.com.cn'
    def page(self):
        for i in range(0, 132):
            if i is 0:
                url = self.__url + '/html/GS/tong-hua-shun/'
            else:
                url = self.__url + '/html/GS/tong-hua-shun/page_%s.html' % i
            self.put(self.__on_page, url, None, {'id': i})
            self.wait()
            time.sleep(1)
    def info(self):
        self.read(self.__on_read)
    def _select(self, conn, sql):
        cursor = conn.cursor()
        cursor.execute(sql)
        result = cursor.fetchall()
        cursor.close()
        return result
    def __on_read(self, conn, args):
        sql = "SELECT * FROM %s WHERE id>5980;" % self.__tb
        result = self._select(conn, sql)
        length = len(result)
        maxlen = 0
        for r in result:
            print('[TohuaShun] id:%s/%d' % (r[0], length))
            info = { 'id': r[0], 'title': r[1], 'href': r[2] }
            self.put(self.__on_info, info['href'], None, info)
            self.wait()
            time.sleep(0.5)
    def _save(self, args):
        print '[TohuaShun] save: %s' % args['title']
        name = args['title']
        content = args['text']
        try:
            file = os.path.join(self.__save_path, name)
            bok = self.create_file(file + '.txt', content)
            if not bok:
                file = os.path.join(self.__save_path, name)
                for i in range(1, 10):
                    bok = self.create_file(file + str(i) + '.txt', content)
                    if bok:
                        break
        except:
            errmsg = '[TohuaShun] error save : title: %s, text: %s' % (args['title'], args['text'])
            self.write(errmsg)
            print errmsg
        finally:
            print(args)
    def __on_save(self, conn, data, args):
        try:
            cursor = conn.cursor()
            sql = "INSERT INTO " + self.__tb + " (title, href)" \
                " VALUES(%s,%s);"
            param = ( args['title'], args['href'] )
            cursor.execute(sql, param)
            cursor.close()
            conn.commit()
        except conn.Error, e:
            err = '[TohuaShun] Error __on_save : %d %s, %s ' % (e[0], e[1], args)
            self.write(err + '\n')
            print(err)
        finally:
            print('[TohuaShun] save id: %d, name: %s' % (args['id'], args['title']))
    def __on_page(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        ps = soup.find_all('p',attrs={'onmouseover':"this.style.background='#E6F3D3'"})
        for p in ps:
            a = p.find('a')
            if not a:
                continue
            args['title'] = a.text
            args['href'] = self.__url + a.get('href')
            self.save(self.__on_save, None, args)
    def __on_info(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        div = soup.find(id="contentgs")
        if not div:
            errmsg = '[TohuaShun] %s: no #contentgs' % (url)
            self.write(errmsg)
            print(errmsg)
            return
        ps = div.find_all('p')
        text = ''
        for p in ps:
            text += p.text
        #
        args['text'] = text

        self._save(args)
    def write(self, text):
        f = codecs.open(self.__tb + '.log', 'w', 'utf-8')
        f.write(text)
        f.close()
    def create_file(self, file, content=u''):
        if os.path.exists(file):
            return False
        path = os.path.dirname(file)
        self.mkdirs(path)
        #
        fp = codecs.open(file, 'w', 'utf-8')
        fp.write(content)
        fp.close()

        return True
    def mkdirs(self, path):
        try:
            os.makedirs(path)
        except:
            pass
#-----------------------
